# -*- coding: utf-8 -*-
import scrapy,requests,json,demjson
from sinanews.sqlHelper import *

class SinaSpider(scrapy.Spider):
    name = 'sina'
    allowed_domains = ['sina.com.cn']
    #采集深度
    spider_page=1
    key_text=["SJ_XW","XJ_TY","XJ_CJ","SJ_KJ","SJ_YL"]
    type_text=["新闻","体育","财经","科技","娱乐"]
    type_index=0
    type_list=[]
    type_list.append("http://roll.news.sina.com.cn/interface/rollnews_ch_out_interface.php?col=89&spec=&type=&ch=01&k=&offset_page=0&offset_num=0&num=60&asc=&page=[*]")#新闻
    type_list.append("http://feed.mix.sina.com.cn/api/roll/get?pageid=13&lid=2503&k=&num=50&page=[*]")#体育
    type_list.append("http://roll.finance.sina.com.cn/finance/zq1/scyj/index_[*].shtml")#财经
    type_list.append("http://feed.mix.sina.com.cn/api/roll/get?pageid=372&lid=2431&k=&num=50&page=[*]")#科技
    type_list.append("http://feed.mix.sina.com.cn/api/roll/get?pageid=382&lid=48&k=&num=50&page=[*]")#娱乐
    offset=0
    page_index=1
    url_list=[]
    news_list=[]
    def parse_list(self,resp):
        res=resp.body
        if resp.meta['type']==0:
            res=res[15:len(res)-1]
            res=res.decode("gb18030")
            res=demjson.decode(res)
            self.url_list=res['list']
        elif resp.meta['type']==1:
            res=eval(res)
            self.url_list=res['result']['data']
        elif resp.meta['type']==2:
            self.url_list=[]
            for each in resp.xpath('//*[@id="Main"]/div[3]/ul'):
                for eac in each.xpath('./li'):
                    self.url_list.append(eac.xpath('./a/@href').extract()[0])
        elif resp.meta['type']==3:
            if res[0:1]!="{":
                res=res[res.index('('),len(res)-res.rindex('catch')-2]
            res=eval(res)
            self.url_list=res['result']['data']
        elif resp.meta['type']==4:
            res=demjson.decode(res)
            self.url_list=res['result']['data']

        print '当前页共有['+str(len(self.url_list))+"]条数据"
        for ul in self.url_list:
            try:
                url=ul['url']
            except:
                url=ul
            url=url.replace("\\","")
            yield scrapy.Request(url,callback=self.parse,dont_filter=True,meta={'type':resp.meta['type']})
    def parse(self, response):
        if response.status==404:
            print '******404错误******'
        save=True
        for each in response.xpath('//*[@class="article"]/img'):
            url=each.xpath('./@src').extract()[0]
            ftype=url[len(url)-4:len(url)].lower()
            if ftype!='.jpg' and ftype!='png':
                save=False
        if save:
            try:
                content=response.xpath('//*[@class="article"]').extract()[0]
                title=response.xpath('//*[@class="main-title"]/text()').extract()[0]
                date=response.xpath('//*[@id="top_bar"]/div/div[2]/span/text()').extract()[0]
                t=response.meta['type']
                category=self.type_text[t]
                _type=self.key_text[t]
                if sqlHelper.selectOne("select title from news where title=%s",(title)):
                    print '重复内容，抛弃'
                else:
                    sqlHelper.update("insert into news(title,category,content,_type,date) values(%s,%s,%s,%s,%s)",(title,category,content,_type,date))
                    self.news_list.append({'sectionKey':_type,'title':title,'context':content})
            except Exception,e:
#                print traceback.print_exc() 
                print '地址'+response.url+"采集异常"
        else:
            print '图片格式不匹配，抛弃！'


    def start_requests(self):
        pages=[]
        t=0
        for ul in self.type_list:
            for i in range(self.spider_page):
                url=ul.replace("[*]",str(i))
                print url
                page=scrapy.Request(url,self.parse_list,meta={'type':t},dont_filter=True)
                pages.append(page)
            t+=1
        return pages
    def close(self,reason):
        print '采集完成,共采集了'+str(len(self.news_list))+'条数据'
        print '开始推送'
        url="http://www.gdefault001.com:8085/platform/doc/collectDoc"
        try:
            res=requests.post(url,data={"result":json.dumps(self.news_list)})
            print "推送结果："
            print res.content
        except:
            print '推送失败！'
            print '与服务器通信失败，可能未开启或故障'
